CLEWs Modelling Dashboard

This notebook is intended for OSeMOSYS users. It allows an OSeMOSYS user to preprocess data files, run models, and visualise results with the click of button.

In [778]:
import pandas as pd
pd.set_option('mode.chained_assignment', None)

#import numpy as np
from IPython.display import HTML
import IPython.core.display as di
#import ipywidgets as widgets
#from ipywidgets import interact, interact_manual

#importing plotly and cufflinks in offline mode
import plotly as py
#import plotly.graph_objs as go
import cufflinks
import plotly.offline as pyo
from plotly.offline import plot, iplot, init_notebook_mode
pyo.init_notebook_mode()
cufflinks.go_offline()
cufflinks.set_config_file(world_readable=True, theme='white')
import os, sys
import subprocess
from tkinter import filedialog
from tkinter import *
from collections import defaultdict

import zipfile
In [779]:
root = Tk()
root.folder =  filedialog.askdirectory()
root.destroy()
In [780]:
HTML('''<script>
code_show=true; 
function code_toggle() {
 if (code_show){
 $('div.input').hide();
 } else {
 $('div.input').show();
 }
 code_show = !code_show
} 
$( document ).ready(code_toggle);
</script>
The raw code for this IPython notebook is by default hidden for easier reading.
To toggle on/off the raw code, click <a href="javascript:code_toggle()">here</a>.''')
Out[780]:
The raw code for this IPython notebook is by default hidden for easier reading. To toggle on/off the raw code, click here.
In [781]:
#folder_path = r'C:\Users\abhis\Downloads\bolivia_test'
#folder_path = sys.path[0] 

data_file = os.path.join(root.folder, 'data.txt') # data_file = os.path.join(sys.path[0], 'data.txt') 
glpsol_command = "glpsol --check -m " + '"' + os.path.join(os.getcwd(),'model_new.txt') + '"' + " -d data.txt --wlp cbcinput.lp --log glpsol_log.txt"
cbc_command = "cbc cbcinput.lp solve -solu cbcoutput.txt"
In [782]:
lines = []
with open(data_file, 'r') as f1:
    for line in f1:
        if not line.startswith(('set MODEper','set MODEx')):
            lines.append(line)
            
with open(data_file, 'w') as f2:
    f2.writelines(lines)
    
parsing = False

data_all = []
data_out = []
data_inp = []

output_table = []
input_table = []

with open(data_file, 'r') as f:
    for line in f:
        if line.startswith('set YEAR'):
            start_year = line.split(' ')[3]
        if line.startswith('set COMMODITY'): # Extracts list of COMMODITIES from data file. Some models use FUEL instead. 
            fuel_list = line.split(' ')[3:-1]
        if line.startswith('set FUEL'): # Extracts list of FUELS from data file. Some models use COMMODITIES instead. 
            fuel_list = line.split(' ')[3:-1]
        if line.startswith('set TECHNOLOGY'):
            tech_list = line.split(' ')[3:-1]
        if line.startswith('set STORAGE'):
            storage_list = line.split(' ')[3:-1]
        if line.startswith('set MODE_OF_OPERATION'):
            mode_list = line.split(' ')[3:-1]

with open(data_file, 'r') as f:
    for line in f:
        if line.startswith(";"):
            parsing = False   
        if parsing:
            if line.startswith('['):
                fuel = line.split(',')[2]
                tech = line.split(',')[1]
            elif line.startswith(start_year):
                years = line.rstrip().split(' ')[0:]
                years = [i.strip(':=') for i in years]
            elif not line.startswith(start_year):
                values = line.rstrip().split(' ')[1:]
                mode = line.split(' ')[0]
                data_out.append(tuple([fuel,tech,mode]))
                data_all.append(tuple([tech,mode]))
                for i in range(0,len(years)):
                    output_table.append(tuple([tech,fuel,mode,years[i],values[i]]))
        if line.startswith('param OutputActivityRatio'):
            parsing = True

with open(data_file, 'r') as f:
    for line in f:
        if line.startswith(";"):
            parsing = False   
        if parsing:
            if line.startswith('['):
                fuel = line.split(',')[2]
                tech = line.split(',')[1]
            elif not line.startswith(start_year):
                mode = line.split(' ')[0]
                data_inp.append(tuple([fuel,tech,mode]))
                data_all.append(tuple([tech,mode]))
                for i in range(0,len(years)):
                    input_table.append(tuple([tech,fuel,mode,years[i],values[i]]))
        if line.startswith('param InputActivityRatio'):
            parsing = True

# For TechnologyToStorage and TechnologyFromStorage 
storage_to = []
storage_from = []

with open(data_file) as f:
    for line in f:
        if line.startswith(";"):
            parsing = False   
        if parsing:
            if line.startswith('['):
                tech = line.split(',')[1]
            elif not line.startswith(mode_list[0]):
                storage = line.split(' ')[0]
                values = line.rstrip().split(' ')[1:]
                for i in range(0,len(mode_list)):
                    if values[i] != '0':
                        storage_to.append(tuple([storage,tech,mode_list[i]]))
                        #data_all.append(tuple([tech,mode_list[i]]))
        if line.startswith('param TechnologyToStorage'):
            parsing = True

with open(data_file) as f:
    for line in f:
        if line.startswith(";"):
            parsing = False   
        if parsing:
            if line.startswith('['):
                tech = line.split(',')[1]
            elif not line.startswith(mode_list[0]):
                storage = line.split(' ')[0]
                values = line.rstrip().split(' ')[1:]
                for i in range(0,len(mode_list)):
                    if values[i] != '0':
                        storage_from.append(tuple([storage,tech,mode_list[i]]))
                        #data_all.append(tuple([tech,mode_list[i]]))
        if line.startswith('param TechnologyFromStorage'):
            parsing = True
            
dict_out = defaultdict(list)
dict_inp = defaultdict(list)
dict_all = defaultdict(list)
dict_stt = defaultdict(list)
dict_stf = defaultdict(list)

for f,t,m in data_out:
    dict_out[f].append((m,t))

for f,t,m in data_inp:
    dict_inp[f].append((m,t))
    
for t,m in data_all:
    if m not in dict_all[t]:
        dict_all[t].append(m)
        
for s,t,m in storage_to:
    dict_stt[s].append((m,t))

for s,t,m in storage_from:
    dict_stf[s].append((m,t))
        
# Open data file and delete line with 'end;' statement
lines = []
with open(data_file, 'r') as f1:
    for line in f1:
        if not line.startswith('end;'):
            lines.append(line)
            
with open(data_file, 'w') as f2:
    f2.writelines(lines)

# Append lines at the end of the data file
with open(data_file, 'a') as file_out: # 'a' to open in 'append' mode
    
    for each in fuel_list:
        if each in dict_out.keys():
            line = 'set MODExTECHNOLOGYperFUELout[' + str(each)+']:=' + str(dict_out[each])
            line = line.replace('),',')').replace('[(',' (').replace(')]',')').replace("'","")
        else:
            line = 'set MODExTECHNOLOGYperFUELout[' + str(each) + ']:='
        file_out.write(line + ';' + '\n')
    
    for each in fuel_list:
        if each in dict_inp.keys():
            line = 'set MODExTECHNOLOGYperFUELin[' + str(each) + ']:=' + str(dict_inp[each])
            line = line.replace('),',')').replace('[(',' (').replace(')]',')').replace("'","")
        else:
            line = 'set MODExTECHNOLOGYperFUELin[' + str(each) + ']:='
        file_out.write(line + ';' + '\n')
    
    for each in tech_list:
        if each in dict_all.keys():
            line = 'set MODEperTECHNOLOGY[' + str(each) + ']:=' + str(dict_all[each]) + '*'
            line = line.replace(',','').replace(':=[',':= ').replace(']*','').replace("'","")
        else:
            line = 'set MODEperTECHNOLOGY[' + str(each) + ']:='
        file_out.write(line + ';' + '\n')
        
    if len(storage_list) > 1:
        for each in storage_list:
            if each in dict_stt.keys():
                line = 'set MODExTECHNOLOGYperSTORAGEto[' + str(each)+']:=' + str(dict_out[each])
                line = line.replace('),',')').replace('[(',' (').replace(')]',')').replace("'","")
            else:
                line = 'set MODExTECHNOLOGYperSTORAGEto[' + str(each) + ']:='
            file_out.write(line + ';' + '\n')
        
    if len(storage_list) > 1:
        for each in storage_list:
            if each in dict_stf.keys():
                line = 'set MODExTECHNOLOGYperSTORAGEfrom[' + str(each)+']:=' + str(dict_out[each])
                line = line.replace('),',')').replace('[(',' (').replace(')]',')').replace("'","")
            else:
                line = 'set MODExTECHNOLOGYperSTORAGEfrom[' + str(each) + ']:='
            file_out.write(line + ';' + '\n')
        
    file_out.write('end;')
In [783]:
try:
    os.remove(os.path.join(root.folder, 'cbcoutput.txt'))
except OSError:
    pass

#subprocess.Popen('start /wait ' + data_filtering, shell=True, cwd=root.folder).wait()
subprocess.Popen('start /wait ' + glpsol_command, shell=True, cwd=root.folder).wait()
subprocess.Popen('start /wait ' + cbc_command, shell=True, cwd=root.folder).wait()

#for file in os.listdir(root.folder):
#    if file.startswith('output'):
#        results_file = os.path.join(root.folder, file)
#        print(results_file)

results_file = os.path.join(root.folder, 'cbcoutput.txt')

# Create \res\csv folder in current working directory if it doesn't already exist
try:
    os.makedirs(os.path.join(root.folder, 'res\csv'))
except FileExistsError:
    pass    
In [784]:
#Read CBC output file
df = pd.read_csv(results_file, sep='\t')
#if str(df.iloc[0]).split(' ')[0] == "Infeasible":
#    print("INFEASIBLE RESULT!  CHECK YOUR PARAMETERS!")
#    exit(0) # Kill the kernel so we don't continue to run...

if str(df.iloc[0]).split(' ')[0] == "Optimal":
    print("Optimal Solution Found.")
df.columns = ['temp']
df['temp'] = df['temp'].str.lstrip(' *\n\t')
df[['temp','value']] = df['temp'].str.split(')', expand=True)
df = df.applymap(lambda x: x.strip() if isinstance(x,str) else x)
df['value'] = df['value'].str.split(' ', expand=True)
df[['parameter','id']] = df['temp'].str.split('(', expand=True)
df['parameter'] = df['parameter'].str.split(' ', expand=True)[1]
df = df.drop('temp', axis=1)
df['value'] = df['value'].astype(float).round(4)
Optimal Solution Found.
In [785]:
#df.columns = ['temp']
#df[['temp','value']] = df['temp'].str.split(')', expand=True)
#df = df.applymap(lambda x: x.strip() if isinstance(x,str) else x)
#df[['temp','parameter']] = df['temp'].str.split(' ', expand=True)
#df[['parameter','id']] = df['parameter'].str.split('(', expand=True)
##df['value'] = df['value'].str.replace(' 0','')
#df['value'] = df['value'].str.split('\s+').str.get(0)
#df = df.drop('temp', axis=1)
#df['value'] = df['value'].astype(float).round(4)
##df = df[~df['value'].str.contains('e-')]
In [786]:
params = df.parameter.unique()
all_params = {}
cols = {'NewCapacity':['r','t','y'],
            'AccumulatedNewCapacity':['r','t','y'], 
            'TotalCapacityAnnual':['r','t','y'],
            'CapitalInvestment':['r','t','y'],
            'AnnualVariableOperatingCost':['r','t','y'],
            'AnnualFixedOperatingCost':['r','t','y'],
            'SalvageValue':['r','t','y'],
            'DiscountedSalvageValue':['r','t','y'],
            'TotalTechnologyAnnualActivity':['r','t','y'],
            'RateOfActivity':['r','l','t','m','y'],
            'RateOfTotalActivity':['r','t','l','y'],
            'Demand':['r','l','f','y'],
            'TotalAnnualTechnologyActivityByMode':['r','t','m','y'],
            'TotalTechnologyModelPeriodActivity':['r','t'],
            'ProductionByTechnologyAnnual':['r','t','f','y'],
            'AnnualTechnologyEmissionByMode':['r','t','e','m','y'],
            'AnnualTechnologyEmission':['r','t','e','y'],
            'AnnualEmissions':['r','e','y'],
            'TotalInputToNewCapacity':['r','f','y'],
            'TotalInputToTotalCapacity':['r','f','y'],
            'TechnologyActivityChangeByModeCostTotal':['r','t','y']}

for each in params:
    df_p = df[df.parameter == each]
    df_p[cols[each]] = df_p['id'].str.split(',',expand=True)
    cols[each].append('value')
    df_p = df_p[cols[each]] # Reorder dataframe to include 'value' as last column
    all_params[each] = pd.DataFrame(df_p) # Create a dataframe for each parameter
    df_p = df_p.rename(columns={'value':each})
    df_p.to_csv(os.path.join(root.folder, 'res\csv', str(each) + '.csv'), index=None) # Print data for each paramter to a CSV file
In [787]:
year_split = []
parsing = False

with open(data_file, 'r') as f:
    for line in f:
        if line.startswith(";"):
            parsing = False   
        if parsing:
            if line.startswith(start_year):
                years = line.rstrip().split(' ')[0:]
                years = [i.strip(':=') for i in years]
            elif not line.startswith(start_year):
                time_slice = line.rstrip().split(' ')[0]
                values = line.rstrip().split(' ')[1:]
                for i in range(0,len(years)):
                    year_split.append(tuple([time_slice,years[i],values[i]]))
        if line.startswith('param YearSplit'):
            parsing = True

df_output = pd.DataFrame(output_table, columns=['t','f','m','y','OutputActivityRatio'])
df_yearsplit = pd.DataFrame(year_split, columns=['l','y','YearSplit'])
df_activity = all_params['RateOfActivity'].rename(columns={'value':'RateOfActivity'})
In [788]:
df_out_ys = pd.merge(df_output, df_yearsplit, on='y')

df_out_ys['OutputActivityRatio'] = df_out_ys['OutputActivityRatio'].astype(float)
df_out_ys['YearSplit'] = df_out_ys['YearSplit'].astype(float)

df_prod = pd.merge(df_out_ys, df_activity, on=['t','m','l','y'])

df_prod['ProductionByTechnologyAnnual'] = df_prod['OutputActivityRatio']*df_prod['YearSplit']*df_prod['RateOfActivity']
df_prod = df_prod.drop(['OutputActivityRatio','YearSplit','RateOfActivity'], axis=1)

df_prod = df_prod.groupby(['r','t','f','y'])['ProductionByTechnologyAnnual'].sum().reset_index()
df_prod['ProductionByTechnologyAnnual'] = df_prod['ProductionByTechnologyAnnual'].astype(float).round(4)

df_prod.to_csv(os.path.join(root.folder, 'res\csv', 'ProductionByTechnologyAnnual.csv'), index=None)
all_params['ProductionByTechnologyAnnual'] = df_prod.rename(columns={'ProductionByTechnologyAnnual':'value'})
In [789]:
df_input = pd.DataFrame(input_table, columns=['t','f','m','y','InputActivityRatio'])

df_in_ys = pd.merge(df_input, df_yearsplit, on='y')

df_in_ys['InputActivityRatio'] = df_in_ys['InputActivityRatio'].astype(float)
df_in_ys['YearSplit'] = df_in_ys['YearSplit'].astype(float)

df_use = pd.merge(df_in_ys, df_activity, on=['t','m','l','y'])

df_use['UseByTechnologyAnnual'] = df_use['InputActivityRatio']*df_use['YearSplit']*df_use['RateOfActivity']
df_use = df_use.drop(['InputActivityRatio','YearSplit','RateOfActivity'], axis=1)

df_use = df_use.groupby(['r','t','f','y'])['UseByTechnologyAnnual'].sum().reset_index()
df_use['UseByTechnologyAnnual'] = df_use['UseByTechnologyAnnual'].astype(float).round(4)

df_use.to_csv(os.path.join(root.folder, 'res\csv', 'UseByTechnologyAnnual.csv'), index=None)
all_params['UseByTechnologyAnnual'] = df_use.rename(columns={'UseByTechnologyAnnual':'value'})
In [790]:
# List of columns for detailed energy tables and figures
years = pd.Series([int(i) for i in years])

name_color_codes = pd.read_csv(os.path.join(os.getcwd(),'name_color_codes.csv'), encoding='latin-1')
det_col = dict([(c,n) for c,n in zip(name_color_codes.code, name_color_codes.name_english)])
color_dict = dict([(n,c) for n,c in zip(name_color_codes.name_english, name_color_codes.colour)])

# List of columns for aggregated energy tables and figures
agg_col = {'Coal':['Coal'],
           'Oil': ['Diesel','HFO','JFL','Crude oil','Petroleum coke'],
           'Gas': ['Natural gas','LNG','LPG'],
           'Hydro': ['Hydro'],
           'Nuclear': ['Nuclear'],
           'Other renewables': ['Biomass','Geothermal','Solar','MSW','Wind'],
           'Net electricity imports': ['Net electricity imports']
          }
In [791]:
def df_filter(df,lb,ub,t_exclude):
    df['t'] = df['t'].str[lb:ub]
    df['value'] = df['value'].astype('float64')
    df = df[~df['t'].isin(t_exclude)].pivot_table(index='y', 
                                          columns='t',
                                          values='value', 
                                          aggfunc='sum').reset_index().fillna(0)
    df = df.reindex(sorted(df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)
    new_df = pd.DataFrame()
    new_df['y'] = years
    new_df['y'] = new_df['y'].astype(int)
    df['y'] = df['y'].astype(int)
    new_df = pd.merge(new_df,df, how='outer', on='y').fillna(0)
    return new_df

def df_plot(df,y_title,p_title):
    return df.iplot(x='y',
                    kind='bar', 
                    barmode='stack',
                    xTitle='Year',
                    yTitle=y_title,
                    color=[color_dict[x] for x in df.columns if x != 'y'],
                    title=p_title,
                    showlegend=True)

Energy sector

In [792]:
gfec_df = all_params['TotalTechnologyAnnualActivity'][all_params['TotalTechnologyAnnualActivity'].t.str.startswith('DEM')].drop('r', axis=1)
gfec_df = gfec_df[~gfec_df.t.str.startswith('DEMAGRSUR')]
gfec_df = gfec_df[~gfec_df.t.str.endswith('CRU')]
gfec_df = df_filter(gfec_df,3,6,['PWR','LVS'])
df_plot(gfec_df,'Energy (PJ)','Gross final energy consumption - by sector')
In [793]:
gfec_fuel_df = all_params['TotalTechnologyAnnualActivity'][all_params['TotalTechnologyAnnualActivity'].t.str.startswith('DEM')].drop('r', axis=1)
gfec_fuel_df = gfec_fuel_df[~gfec_fuel_df.t.str.startswith('DEMPWR')]
gfec_fuel_df = df_filter(gfec_fuel_df,6,9,['SUR','WND','HYD','SOL','GEO','GWT','CRU'])
df_plot(gfec_fuel_df,'Energy (PJ)','Gross final energy consumption - by fuel')
In [794]:
for sector in ['AGR','COM','IND','OTH','RES','TRA']:
    gfec_sector_df = all_params['TotalTechnologyAnnualActivity'][all_params['TotalTechnologyAnnualActivity'].t.str.startswith('DEM'+sector)].drop('r', axis=1)
    gfec_sector_df = df_filter(gfec_sector_df,6,9,['SUR','WND','HYD','SOL','GEO','GWT','CRU'])
    df_plot(gfec_sector_df,'Energy (PJ)','Gross final energy consumption - ' + det_col[sector] + ' sector' )
In [795]:
# Power generation capacity (detailed)
cap_df = all_params['TotalCapacityAnnual'][all_params['TotalCapacityAnnual'].t.str.startswith('PWR')].drop('r', axis=1)
cap_df = df_filter(cap_df,3,6,['CNT','TRN','CST','CEN','SOU','NOR'])
df_plot(cap_df,'Gigawatts (GW)','Power Generation Capacity (Detail)')
In [796]:
# Power generation capacity (Aggregated)
cap_agg_df = pd.DataFrame(columns=agg_col)
cap_agg_df.insert(0,'y',cap_df['y'])
cap_agg_df  = cap_agg_df.fillna(0.00)

for each in agg_col:
    for tech_exists in agg_col[each]:
        if tech_exists in cap_df.columns:
            cap_agg_df[each] = cap_agg_df[each] + cap_df[tech_exists]
            cap_agg_df[each] = cap_agg_df[each].round(2)

cap_agg_df = cap_agg_df.loc[:,(cap_agg_df != 0).any(axis=0)]
df_plot(cap_agg_df,'Gigawatts (GW)','Power Generation Capacity (Aggregate)')
In [797]:
#Power generation (Detailed)
gen_df = all_params['ProductionByTechnologyAnnual'][(all_params['ProductionByTechnologyAnnual'].t.str.startswith('PWR') |
                                                     all_params['ProductionByTechnologyAnnual'].t.str.startswith('IMP')) & 
                                                   all_params['ProductionByTechnologyAnnual'].f.str.startswith('ELC')].drop('r', axis=1)

gen_df = df_filter(gen_df,3,6,['TRN'])

gen_df['Net electricity imports'] = 0

ele_exp_df = all_params['TotalTechnologyAnnualActivity'][all_params['TotalTechnologyAnnualActivity'].t.str.startswith('EXPELC')].drop('r', axis=1)

if not ele_exp_df.empty:
    ele_exp_df = df_filter(ele_exp_df,3,6,['TRN']).rename(columns={'Electricity':'Electricity exports'})
    gen_df = gen_df.merge(ele_exp_df)
    gen_df['Net electricity imports'] = gen_df['Net electricity imports'] - gen_df['Electricity exports']
    gen_df.drop('Electricity exports', axis=1, inplace=True)

if 'Electricity' in gen_df.columns:
    gen_df['Net electricity imports'] = gen_df['Net electricity imports'] - gen_df['Electricity']
    gen_df.drop('Electricity', axis=1, inplace=True)
    #gen_df = gen_df.rename(columns={'Electricity':'Electricity imports'})

#df_plot(gen_df,'Petajoules (PJ)','Power Generation (Detail)')
gen_df.iplot(x='y',
            kind='bar', 
            barmode='relative',
            xTitle='Year',
            yTitle='Petajoules (PJ)',
            color=[color_dict[x] for x in gen_df.columns if x != 'y'],
            title='Power Generation (Detail)',
            showlegend=True)
In [798]:
# Power generation (Aggregated)
gen_agg_df = pd.DataFrame(columns=agg_col)
gen_agg_df.insert(0,'y',gen_df['y'])
gen_agg_df  = gen_agg_df.fillna(0.00)

for each in agg_col:
    for tech_exists in agg_col[each]:
        if tech_exists in gen_df.columns:
            gen_agg_df[each] = gen_agg_df[each] + gen_df[tech_exists]
            gen_agg_df[each] = gen_agg_df[each].round(2)

gen_agg_df = gen_agg_df.loc[:,(gen_agg_df != 0).any(axis=0)]

gen_agg_df.iplot(x='y',
            kind='bar', 
            barmode='relative',
            xTitle='Year',
            yTitle='Petajoules (PJ)',
            color=[color_dict[x] for x in gen_agg_df.columns if x != 'y'],
            title='Power Generation (Aggregate)',
            showlegend=True)
In [799]:
# Fuel use for power generation
gen_use_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].t.str.startswith('DEMPWR')].drop('r', axis=1)
gen_use_df = df_filter(gen_use_df,6,9,['SUR'])
df_plot(gen_use_df,'Petajoules (PJ)','Power Generation (Fuel use)')
In [800]:
#Power sector emissions
emissions_df = all_params['AnnualTechnologyEmission'][all_params['AnnualTechnologyEmission'].t.str.startswith('DEM')].drop('r', axis=1)
emissions_fuel_df = df_filter(emissions_df,6,9,[])

emissions_df = all_params['AnnualTechnologyEmission'][all_params['AnnualTechnologyEmission'].t.str.startswith('DEM')].drop('r', axis=1)
emissions_sector_df = df_filter(emissions_df,3,6,[])

df_plot(emissions_fuel_df,'Million tonnes of CO2','CO2 emissions by fuel')
df_plot(emissions_sector_df,'Million tonnes of CO2','CO2 emissions by sector')
In [801]:
#Domestic fuel production
dom_prd_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].t.str.startswith('MIN')|
                                                       all_params['ProductionByTechnologyAnnual'].t.str.startswith('RNW')].drop('r', axis=1)
dom_prd_df = df_filter(dom_prd_df,3,6,[])

for each in dom_prd_df.columns:
    if each in ['Land','Water', 'Precipitation']: #,'Hydro','Solar','Wind','Geothermal'
        dom_prd_df = dom_prd_df.drop(each, axis=1)

try:
    df_plot(dom_prd_df,'Petajoules (PJ)','Domestic energy production')
except:
    print("No domestic fuel production.")
In [802]:
#Energy imports
ene_imp_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].t.str.startswith('IMP')].drop('r', axis=1)
ene_imp_df = df_filter(ene_imp_df,3,6,[])
if not ene_imp_df.empty:
    df_plot(ene_imp_df,'Petajoules (PJ)','Energy imports')
In [803]:
#Energy exports
ene_exp_df = all_params['TotalTechnologyAnnualActivity'][all_params['TotalTechnologyAnnualActivity'].t.str.startswith('EXP')].drop('r', axis=1)
ene_exp_df = df_filter(ene_exp_df,3,6,[])

if len(ene_exp_df.columns) > 1:
    df_plot(ene_exp_df,'Petajoules (PJ)','Energy exports')
In [804]:
cap_cos_df = all_params['CapitalInvestment'][all_params['CapitalInvestment'].t.str.startswith('PWR')].drop('r', axis=1)
cap_cos_df = df_filter(cap_cos_df,3,6,['TRN'])
df_plot(cap_cos_df,'Million $','Capital Investment')
In [805]:
ele_cos_df = pd.DataFrame(columns=['Total capital investment', 'Capital costs'])
ele_cos_df.insert(0,'y',years)
ele_cos_df['Total capital investment'] = cap_cos_df.iloc[:,1:].sum(axis=1)
ele_cos_df['Capital costs'] = 0
ele_cos_df = ele_cos_df.fillna(0)

R = 0.1 # Discount rate
n = 30 # Amortization period
cap_exist_total = 500 # Payments on existing capacity (legacy costs)


for i in ele_cos_df['y']:
    for j in ele_cos_df['y']:
        if i < j + n and i >= j:
            ele_cos_df.loc[ele_cos_df['y']==i,'Capital costs'] = ele_cos_df.loc[ele_cos_df['y']==i,'Capital costs'] + (((ele_cos_df.loc[ele_cos_df['y']==j,'Total capital investment'].iloc[0])*R)/(1-(1+R)**(-n)))

ele_cos_df.drop('Total capital investment', axis=1, inplace=True)
In [806]:
cap_exist_values = {}

for year in years:
    if cap_exist_total - ((cap_exist_total/n)*(year - int(start_year))) > 0:
        cap_exist_values[year] = cap_exist_total - ((cap_exist_total/n)*(year - int(start_year)))
    else:
        cap_exist_values[year] = 0

ele_cos_df['Legacy costs'] = ele_cos_df['y'].map(cap_exist_values)
ele_cos_df['Capital costs'] += ele_cos_df['Legacy costs'] 
ele_cos_df = ele_cos_df.drop('Legacy costs', axis=1)
In [807]:
fix_cos_df = all_params['AnnualFixedOperatingCost'][all_params['AnnualFixedOperatingCost'].t.str.startswith('PWR')].drop('r', axis=1)
fix_cos_df = df_filter(fix_cos_df,3,6,['TRN'])

var_cos_df = all_params['AnnualVariableOperatingCost'][all_params['AnnualVariableOperatingCost'].t.str.startswith('PWR')|
                                                       all_params['AnnualVariableOperatingCost'].t.str.startswith('EXPELC')].drop('r', axis=1)
var_cos_df = df_filter(var_cos_df,3,6,['TRN'])

dis_cos_df = all_params['AnnualVariableOperatingCost'][all_params['AnnualVariableOperatingCost'].t.str.startswith('DEMPWR')].drop('r', axis=1)
dis_cos_df = df_filter(dis_cos_df,6,9,[])

dom_val_df = all_params['AnnualVariableOperatingCost'][all_params['AnnualVariableOperatingCost'].t.str.startswith('MIN')|
                                                       all_params['AnnualVariableOperatingCost'].t.str.startswith('RNW')].drop('r', axis=1)
dom_val_df = df_filter(dom_val_df,3,6,[])
for each in dom_val_df.columns:
    if each in ['Land','Water','Geothermal','Hydro','Solar','Wind', 'Precipitation']:
        dom_val_df = dom_val_df.drop(each, axis=1)
        
imp_val_df = all_params['AnnualVariableOperatingCost'][all_params['AnnualVariableOperatingCost'].t.str.startswith('IMP')].drop('r', axis=1)
imp_val_df = df_filter(imp_val_df,3,6,[])

exp_val_df = all_params['AnnualVariableOperatingCost'][all_params['AnnualVariableOperatingCost'].t.str.startswith('EXP')].drop('r', axis=1)
exp_val_df = df_filter(exp_val_df,3,6,[])
In [808]:
temp_col_list = []
temp_col_list = dom_val_df.columns

if len(imp_val_df.columns) > 1:
    temp_col_list = temp_col_list.append(imp_val_df.columns)

if len(exp_val_df.columns) > 1:
    temp_col_list = temp_col_list.append(exp_val_df.columns)

fue_val_df = pd.DataFrame(columns=list(set(temp_col_list)))
fue_val_df['y'] = years
fue_val_df = fue_val_df.reindex(sorted(fue_val_df.columns),axis=1).set_index('y').reset_index()

fue_val_df = fue_val_df.fillna(0)

for each in dom_val_df.columns:
    if each != 'y':
        fue_val_df[each] = dom_val_df[each]
        fue_val_df = fue_val_df.fillna(0)

for each in imp_val_df.columns :
    if each != 'y' and len(imp_val_df.columns) > 1:
        fue_val_df[each] = fue_val_df[each] + imp_val_df[each]
        fue_val_df = fue_val_df.fillna(0)

for each in exp_val_df.columns:
    if each != 'y' and len(ene_exp_df.columns) > 1:
        fue_val_df[each] = fue_val_df[each] + exp_val_df[each]
        fue_val_df = fue_val_df.fillna(0)
In [809]:
temp_col_list = []
temp_col_list = dom_prd_df.columns
if len(ene_imp_df.columns) > 1:
    temp_col_list = temp_col_list.append(ene_imp_df.columns)
if len(ene_exp_df.columns) > 1:
    temp_col_list = temp_col_list.append(ene_exp_df.columns)

fue_prd_df = pd.DataFrame(columns=list(set(temp_col_list)))
fue_prd_df['y'] = years
fue_prd_df = fue_prd_df.reindex(sorted(fue_prd_df.columns),axis=1).set_index('y').reset_index()

fue_prd_df = fue_prd_df.fillna(0)

for each in dom_prd_df.columns:
    if each != 'y':
        fue_prd_df[each] = dom_prd_df[each]
        fue_prd_df = fue_prd_df.fillna(0)

for each in ene_imp_df.columns:
    if each != 'y' and len(ene_imp_df.columns) > 1:
        fue_prd_df[each] = fue_prd_df[each] + ene_imp_df[each]
        fue_prd_df = fue_prd_df.fillna(0)

for each in ene_exp_df.columns:
    if each != 'y' and len(ene_exp_df.columns) > 1:
        fue_prd_df[each] = fue_prd_df[each] - ene_exp_df[each]
        fue_prd_df = fue_prd_df.fillna(0)
In [810]:
for df in [fue_val_df, fue_prd_df]:
    #df['Diesel'] = df['Crude oil'].mul(0.3755) + df['Diesel']
    #df['HFO'] = df['Crude oil'].mul(0.0171) + df['HFO']
    #df.drop('Crude oil', axis=1, inplace=True)
    
    df['Diesel'] = df['Diesel']
    df['HFO'] = df['HFO']
    #df.drop('Crude oil', axis=1, inplace=True)
In [811]:
fue_cos_df = pd.DataFrame(columns=list(set(temp_col_list)))
fue_cos_df['y'] = years

fue_cos_df = (fue_val_df/fue_prd_df)*gen_use_df
fue_cos_df = fue_cos_df.fillna(0)

fue_cos_df = fue_cos_df.reindex(sorted(fue_cos_df.columns),axis=1).set_index('y').reset_index()
fue_cos_df['y'] = years
In [812]:
ele_cos_df['Electricity generation'] = gen_df.iloc[:,1:].sum(axis=1)/3.6
ele_cos_df['Capital costs'] = ele_cos_df['Capital costs']/ele_cos_df['Electricity generation']
ele_cos_df['Fixed costs'] = fix_cos_df.iloc[:,1:].sum(axis=1)/ele_cos_df['Electricity generation']
ele_cos_df['Variable costs'] = var_cos_df.iloc[:,1:].sum(axis=1)/ele_cos_df['Electricity generation']
ele_cos_df['Fuel distribution costs'] = dis_cos_df.iloc[:,1:].sum(axis=1)/ele_cos_df['Electricity generation']
ele_cos_df['Fuel costs'] = fue_cos_df.iloc[:,1:].sum(axis=1)/ele_cos_df['Electricity generation']
In [813]:
ele_cos_df.drop('Electricity generation',axis=1,inplace=True)
In [814]:
ele_cos_df.iplot(kind='bar',barmode='relative',x='y',title='Cost of electricity generation ($/MWh)')
In [815]:
# Land use

regions = {"TIG": 'TIG',
           "AFA": 'AFA',
           "AMH": 'AMH',
           "BEN": 'BEN',
           #"DIR": '0',
           #"HAR",
           #"ADD",
           "GAM": 'GAM',
           "ORO": 'ORO',
           "SOM": 'SOM',
           "SOU": 'SOU',
}

crops = {}
for each in det_col.keys():
    if each.startswith('CP'):
        crops[each] = det_col[each]
        
# DEFINE THAT THE NAMING CONVENTION FILE NEED TO LIST CROPS ALPHABETICAL

water_supply = {'I':'Irrigated',
                'R':'Rain-fed'}

input_level = {'L':'Low',
               'I':'Intermediate',
               'H':'High'}

mode_crop_combo_input = pd.read_csv(os.path.join(os.getcwd(),'mode_list.csv'), encoding='latin-1')
mode_crop_combo = dict([(m,c) for m,c in zip(mode_crop_combo_input['mode'], mode_crop_combo_input['crop_combo'])])

Land use sector

In [816]:
crops_total_df = all_params['TotalAnnualTechnologyActivityByMode'][all_params['TotalAnnualTechnologyActivityByMode'].t.str.startswith('LNDAGR')].drop('r', axis=1)
#crops_total_df['land_use'] = crops_total_df.m.map(mode_search)

crops_total_df['m'] = crops_total_df['m'].astype(int)
crops_total_df['crop_combo'] = crops_total_df['m'].map(mode_crop_combo)
crops_total_df['land_use'] = crops_total_df['crop_combo'].str[0:4]
crops_total_df.drop(['m','crop_combo'], axis=1, inplace=True)

crops_total_df = crops_total_df[crops_total_df['land_use'].str.startswith('CP')]
crops_total_df = crops_total_df.pivot_table(index='y', 
                                            columns='land_use',
                                            values='value', 
                                            aggfunc='sum').reset_index().fillna(0)
crops_total_df = crops_total_df.reindex(sorted(crops_total_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col).astype('float64')

if not crops_total_df.empty:
    df_plot(crops_total_df,'Land area (1000 sq.km.)','Area by crop')
In [817]:
land_total_df = all_params['TotalAnnualTechnologyActivityByMode'][all_params['TotalAnnualTechnologyActivityByMode'].t.str.startswith('LNDAGR')].drop('r', axis=1)

land_total_df['m'] = land_total_df['m'].astype(int)
land_total_df['crop_combo'] = land_total_df['m'].map(mode_crop_combo)
land_total_df['land_use'] = land_total_df['crop_combo'].str[0:4]
land_total_df.drop(['m','crop_combo'], axis=1, inplace=True)

land_total_df = land_total_df.pivot_table(index='y', 
                                          columns='land_use',
                                          values='value', 
                                          aggfunc='sum').reset_index().fillna(0)
land_total_df['AGR'] = 0

for crop in crops:
    if crop in land_total_df.columns:
        land_total_df['AGR'] += land_total_df[crop]
        land_total_df.drop(crop, axis=1, inplace=True)
land_total_df = land_total_df.reindex(sorted(land_total_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col).astype('float64')


if not land_total_df.empty:
    df_plot(land_total_df,'Land area (1000 sq.km.)','Area by land cover type')
In [818]:
for each in regions.keys():
    crops_region_df = all_params['TotalAnnualTechnologyActivityByMode'][all_params['TotalAnnualTechnologyActivityByMode'].t.str.startswith('LNDAGR')].drop('r', axis=1)
    crops_region_df = crops_region_df[crops_region_df.t.str[6:9] == each]
    
    crops_region_df['m'] = crops_region_df['m'].astype(int)
    crops_region_df['crop_combo'] = crops_region_df['m'].map(mode_crop_combo)
    crops_region_df['land_use'] = crops_region_df['crop_combo'].str[0:4]
    crops_region_df.drop(['m','crop_combo'], axis=1, inplace=True)
    
    crops_region_df = crops_region_df[crops_region_df['land_use'].str.startswith('CP')]
    crops_region_df = crops_region_df.pivot_table(index='y', 
                                            columns='land_use',
                                            values='value', 
                                            aggfunc='sum').reset_index().fillna(0)
    crops_region_df = crops_region_df.reindex(sorted(crops_region_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col).astype('float64')

    if not crops_region_df.empty:
        df_plot(crops_region_df,'Land area (1000 sq.km.)','Area by crop (' + regions[each] + ' region)')
In [819]:
for each in regions.keys():
    land_cluster_df = all_params['TotalAnnualTechnologyActivityByMode'][all_params['TotalAnnualTechnologyActivityByMode'].t.str.startswith('LNDAGR')].drop('r', axis=1)
    land_cluster_df = land_cluster_df[land_cluster_df.t.str[6:9] == each]

    land_cluster_df['m'] = land_cluster_df['m'].astype(int)
    land_cluster_df['crop_combo'] = land_cluster_df['m'].map(mode_crop_combo)
    land_cluster_df['land_use'] = land_cluster_df['crop_combo'].str[0:4]
    land_cluster_df.drop(['m','crop_combo'], axis=1, inplace=True)

    land_cluster_df['value'] = land_cluster_df['value'].astype('float64')
    land_cluster_df = land_cluster_df.pivot_table(index='y', 
                                          columns='land_use',
                                          values='value', 
                                          aggfunc='sum').reset_index().fillna(0)
    land_cluster_df['AGR'] = 0
    
    for crop in crops:
        if crop in land_cluster_df.columns:
            land_cluster_df['AGR'] += land_cluster_df[crop]
            land_cluster_df.drop(crop, axis=1, inplace=True)
    
    land_cluster_df = land_cluster_df.reindex(sorted(land_cluster_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)
    if not land_cluster_df.empty:
        df_plot(land_cluster_df,'Land area (1000 sq.km.)','Area by land cover type (' + regions[each] + ' region)')
In [820]:
for each in water_supply.keys():
    crops_ws_df = all_params['TotalAnnualTechnologyActivityByMode'][all_params['TotalAnnualTechnologyActivityByMode'].t.str.startswith('LNDAGR')].drop('r', axis=1)
    
    crops_ws_df['m'] = crops_ws_df['m'].astype(int)
    crops_ws_df['crop_combo'] = crops_ws_df['m'].map(mode_crop_combo)
    #print(crops_ws_df.head())
    crops_ws_df = crops_ws_df[(crops_ws_df.crop_combo.str.startswith('CP')) & (crops_ws_df.crop_combo.str[5:6] == each)]
    crops_ws_df['land_use'] = crops_ws_df['crop_combo'].str[0:4]
    crops_ws_df.drop(['m','crop_combo'], axis=1, inplace=True)
    
    crops_ws_df = crops_ws_df.pivot_table(index='y', 
                                          columns='land_use',
                                          values='value', 
                                          aggfunc='sum').reset_index().fillna(0)
    crops_ws_df = crops_ws_df.reindex(sorted(crops_ws_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)
    if len(crops_ws_df.columns) > 1:
        df_plot(crops_ws_df,'Land area (1000 sq.km.)','Area by crop (' + water_supply[each] + ')')
In [821]:
crops_prod_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].f.str.startswith('CRP')].drop('r', axis=1)
crops_prod_df['f'] = crops_prod_df['f'].str[3:7]
crops_prod_df['value'] = crops_prod_df['value'].astype('float64')

crops_prod_df = crops_prod_df.pivot_table(index='y', 
                                          columns='f',
                                          values='value',
                                          aggfunc='sum').reset_index().fillna(0)
crops_prod_df = crops_prod_df.reindex(sorted(crops_prod_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)
crops_prod_df['y'] = years

if len(crops_prod_df.columns) > 1:
    df_plot(crops_prod_df,'Production (Million tonnes)','Crop production')
In [822]:
lvs_prod_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].f.str.startswith('LVS')].drop('r', axis=1)
lvs_prod_df = lvs_prod_df.loc[lvs_prod_df['f'].str.contains('PRD')]
lvs_prod_df['value'] = lvs_prod_df['value'].astype('float64')

lvs_prod_df = lvs_prod_df.pivot_table(index='y', 
                                          columns='f',
                                          values='value',
                                          aggfunc='sum').reset_index().fillna(0)
lvs_prod_df = lvs_prod_df.reindex(sorted(lvs_prod_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)
lvs_prod_df['y'] = years

if len(lvs_prod_df.columns) > 1:
    df_plot(lvs_prod_df,'Population (Million head)','Livestock production')
In [827]:
crops_yield_df = crops_prod_df/crops_total_df
crops_yield_df['y'] = years

#print(crops_yield_df.loc[crops_yield_df['y'] == min(years)].transpose().to_string(header=False, index=False))

crops_yield_df.loc[:,crops_yield_df.columns != 'y'] = crops_yield_df.loc[:,crops_yield_df.columns != 'y'].mul(10)


if len(crops_yield_df.columns) > 1:
    crops_yield_df.iplot(x='y',
                     mode='lines+markers', 
                     xTitle='Year',
                     yTitle='Yield (t/ha)',
                     size=10,
                     color=[color_dict[x] for x in crops_yield_df.columns if x != 'y'],
                     title='Yield (tonnes/hectare)')#

Water sector

In [824]:
wat_dem_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].f.str[0:6].isin(['AGRWAT','PUBWAT','PWRWAT','INDWAT','LVSWAT'])].drop('r', axis=1)
wat_dem_df['f'] = wat_dem_df['f'].str[0:3]
wat_dem_df['value'] = wat_dem_df['value'].astype('float64')
wat_dem_df = wat_dem_df.pivot_table(index='y', 
                                          columns='f',
                                          values='value',
                                          aggfunc='sum').reset_index().fillna(0)

pwrwat_df = all_params['ProductionByTechnologyAnnual'][(all_params['ProductionByTechnologyAnnual'].t.str.startswith('PWR')) & 
                                                   all_params['ProductionByTechnologyAnnual'].f.str.startswith('WTR')].drop('r', axis=1)
pwrwat_df['f'] = pwrwat_df['f'].str[0:3]
pwrwat_df['value'] = pwrwat_df['value'].astype('float64')

pwrwat_df = pwrwat_df.pivot_table(index='y', 
                                          columns='f',
                                          values='value',
                                          aggfunc='sum').reset_index().fillna(0)

if not pwrwat_df.empty:
    wat_dem_df['PWR_WAT'] = wat_dem_df['PWR'] - pwrwat_df['WTR']
    
wat_dem_df = wat_dem_df.reindex(sorted(wat_dem_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)

if 'Power sector' in wat_dem_df.columns:
    wat_dem_df.rename(columns={'Power sector':'Power sector (withdrawal)'}, inplace=True)

wat_dem_df = wat_dem_df.loc[:, (wat_dem_df != 0).any(axis=0)]    

wat_dem_df.iplot(x='y',
             kind='bar', 
             barmode='stack',
             xTitle='Year',
             yTitle='Billion m3',
             color=[color_dict[x] for x in wat_dem_df.columns if x != 'y'],
             title='Water Demand',
             showlegend=True)
In [825]:
wat_source_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].f.str[0:6].isin(['AGRWAT','PUBWAT','PWRWAT'])].drop('r', axis=1)
wat_source_df = df_filter(wat_source_df,6,9,[])

df_plot(wat_source_df, 'Billion m3', 'Water withdrawal by source')
In [826]:
wat_bal_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].f.str.startswith('WTR')].drop('r', axis=1)
wat_bal_df['f'] = wat_bal_df['f'].str[3:6]
wat_bal_df['value'] = wat_bal_df['value'].astype('float64')
wat_bal_df = wat_bal_df.pivot_table(index='y', 
                                          columns='f',
                                          values='value',
                                          aggfunc='sum').reset_index().fillna(0)
wat_bal_df = wat_bal_df.reindex(sorted(wat_bal_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)
wat_bal_df['Irrigation'] = wat_dem_df['Agriculture']
wat_bal_df['y'] = years
for each in wat_bal_df.columns:
    if each in ['Evapotranspiration', 'Groundwater recharge', 'Surface water run-off']:
        wat_bal_df[each] = wat_bal_df[each].mul(-1)

if 'Power sector (withdrawal)' in wat_bal_df:
    wat_bal_df['Surface water run-off'] = wat_bal_df['Surface water run-off'] + wat_dem_df['Power sector (withdrawal)'] - wat_dem_df['Power sector (consumptive use)']

wat_bal_df.iplot(x='y',
             kind='bar', 
             barmode='relative',
             xTitle='Year',
             yTitle='Billion m3',
             color=[color_dict[x] for x in wat_bal_df.columns if x != 'y'],
             title='Water balance')